import requests
from bs4 import BeautifulSoup
import pandas as pd

# 第一步：网页爬取
url = 'https://lz.esf.fang.com'
response = requests.get(url)
soup = BeautifulSoup(response.text, 'html.parser')

listings = soup.find_all('dl', class_='list')

data = []

# 调试信息：检查listings是否正确
print(f'Found {len(listings)} listings.')

for listing in listings:
    try:
        title = listing.find('p', class_='title').text.strip()
        price = listing.find('span', class_='price').text.strip()
        area = listing.find('p', class_='area').text.strip()
        rooms = listing.find('p', class_='room').text.strip()
        location = listing.find('p', class_='location').text.strip()

        data.append({
            'title': title,
            'price': price,
            'area': area,
            'rooms': rooms,
            'location': location
        })
    except AttributeError as e:
        print(f'Error parsing listing: {e}')
        continue

# 打印数据以检查爬取是否成功
print(data)

# 第二步：数据清洗
if data:
    df = pd.DataFrame(data)

    # 检查DataFrame是否正确
    print(df.head())

    # 修正正则表达式的转义问题
    df['price'] = df['price'].str.extract(r'(\d+)', expand=False).astype(float)
    df['area'] = df['area'].str.extract(r'(\d+)', expand=False).astype(float)

    # 第三步：数据分析
    # 基本统计
    print(df.describe())

    # 第四步：特征选择
    # 假设有更多特征，可以使用相关性或其他技术选择显著特征

    # 第五步：建立模型（以线性回归为例）
    from sklearn.model_selection import train_test_split
    from sklearn.linear_model import LinearRegression

    # 需要将rooms转换为数值类型
    df['rooms'] = df['rooms'].str.extract(r'(\d)', expand=False).astype(float)

    X = df[['area', 'rooms']]  # 示例特征选择
    y = df['price']

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

    model = LinearRegression()
    model.fit(X_train, y_train)

    print(f'模型系数: {model.coef_}')
    print(f'模型截距: {model.intercept_}')

    # 第六步：保存数据
    df.to_csv('lanzhou_second_hand_houses.csv', index=False)
else:
    print('No data to process.')
